﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Web;

using SDT.CrawlSystem.Servers.Filters;
using SDT.CrawlSystem.Servers;
using HtmlAgilityPack;
using System.Text.RegularExpressions;



namespace DateExtract
{
    
    public sealed partial class WebCrawler 
    {
        private const string XML_RESULT_TEMPLATE =
            "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\r\n<_{0}>\r\n\t<TF><![CDATA[{1}]]></TF>\r\n\t <LF><![CDATA[{7}]]></LF>\r\n\t<CF><![CDATA[{2}]]></CF>\r\n\t<PF>{3}</PF>\r\n\t<MF>{4}</MF>\r\n\t<TXT><![CDATA[{5}]]></TXT>\r\n</_{6}>";
       

        #region Properties from data file
        private string dataFile = null;
        public string DataFile
        {
            get { return dataFile; }
            set { dataFile= value; }
        }

       
        #endregion

        #region Constructors
        internal WebCrawler(string dataFile)
            : base()
        {
           
            this.dataFile = dataFile;

            LoadData();
        }
        #endregion

        #region Download webpage
        private class RequestState
        {
            // Holds the request
            public WebRequest Request;

            public WebWorkUnit WorkUni;

            public RequestState(WebRequest request, WebWorkUnit workUnit)
            {
                Request = request;
                WorkUni = workUnit;
            }
        }

        public string Title { get; set; }
        public string Lead { get; set; }
        public string html { get; set; }
        public string Date { get; set; }
        public string Txt { get; set; }
        public string Content { get; set; }
        public string RawContent { get; set; }
      

        public void DownloadWebsite(string url)
        {
            UrlLoader temp=  (UrlLoader)UrlLoaders[0];
            WebWorkUnit wtemp = new WebWorkUnit(1, 1, temp,url);

            try
            {
                var r = (HttpWebRequest)HttpWebRequest.Create(wtemp.Url);
                r.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8 GTB6 (.NET CLR 3.5.30729)";
                r.Method = "GET";

                var state = new RequestState(r, wtemp);
                var result = r.BeginGetResponse(new AsyncCallback(DownloadCompletedCallback), state);

              
            }
            catch (Exception ex)
            {
            }
        }

        private void DownloadCompletedCallback(IAsyncResult result)
        {
            var state = result.AsyncState as RequestState;
            try
            {
                var req = state.Request as WebRequest;

                #region Get the Response
                var res = req.EndGetResponse(result) as HttpWebResponse;
                var s = res.GetResponseStream() as Stream;
                //, Encoding.GetEncoding("utf-8")
                //, Encoding.GetEncoding("iso-8859-1")
                //var r = new StreamReader(s);
                var r = new StreamReader(s, state.WorkUni.TheUFilter.Enc);
                

                lock (state.WorkUni)
                {
                    state.WorkUni.Ready = true;
                    string RawHtml = r.ReadToEnd();
                    //save data
                    RawContent = RawHtml;
                    //
                    SaveResult(state.WorkUni, RawHtml);
                   
                }
                #endregion

                #region Release resources
                res.Close();
                s.Close();
                r.Close();
                #endregion
            }
            
            #endregion
           
            catch (Exception ex)
            {
               
            }
           
        }
            
      

        #region Result handler

        private bool SaveResult(WebWorkUnit w, string RawHtml)
        {
            var _w = w as WebWorkUnit;

            string txt = null;
            var hf = _w.TheUFilter.HFilter.DoFilter(RawHtml, txt, ref txt,true);    
            
            txt = HttpUtility.HtmlDecode(txt);
            this.Txt = txt;
            html = hf.ToString();

            #region Do title filter
            string tf = null;
            var temp = _w.TheUFilter.TFilter.DoFilter(_w.TheUFilter.TFilter.DataSource == Filter.Regions.WEB ? RawHtml : hf, txt, ref tf,true);

            if (tf == null)
                tf = temp == null ? "" : temp;
           // tf = HttpUtility.HtmlDecode(tf);
            Title = StripHTML(tf.ToString());           

            #endregion

            #region Do lead filter
            string lf = null;
            temp = _w.TheUFilter.LFilter.DoFilter(_w.TheUFilter.LFilter.DataSource == Filter.Regions.WEB ? RawHtml : hf,txt, ref lf,true);
            if (lf == null)
                lf = temp == null ? "" : temp;
           // lf = HttpUtility.HtmlDecode(lf);
            Lead = StripHTML(lf.ToString());
            
           
            #endregion

            #region Date time
            var df = _w.TheUFilter.DFilter.DoFilter(_w.TheUFilter.DFilter.DataSource == Filter.Regions.WEB
                ? RawHtml : hf, txt);
            Date = df.ToString();
            
            #endregion            


            #region Do content filter
            string cf = null;
            temp = null;

            temp = _w.TheUFilter.ConFilter.DoFilter(_w.TheUFilter.ConFilter.DataSource == Filter.Regions.WEB ? RawHtml : hf, txt, ref cf, true);
           
            //if (cf == null)
                //cf = temp == null ? "" : temp;
            if (temp != null)
                cf = temp;

            //cf = HttpUtility.HtmlDecode(cf);

            //Content = cf;

            //string contentRawHtml = null;
            //contentRawHtml = cf;           

            string contentRawHtml = null;
            contentRawHtml = cf;
            //remove microsoft word character
           // cf = MSWordFilter.CleanWordHtml(cf);
            //remove attribute and tag 

            cf = CleanHtmlContentCafefLand(cf, _w.TheUFilter.RemoveTag, _w.TheUFilter.RemoveAttribute);

          

            //replace tag 
            //example : <div> replace by none character
            cf = StripTagsAndAttributes(cf, _w.TheUFilter.AllowTag);

            //cf = HttpUtility.HtmlDecode(cf);
            Content = cf;
            //var _img = new ImgDes();
            //IList<string> l = new List<string>();
            //if (!string.IsNullOrEmpty(cf))
            //{

            //    _img.BaseURL = _w.TheUFilter.Prefix;
            //    l = _img.GetUrlAttributeSrc(_img.BaseURL, cf);
            //    if (l != null && l.Count > 0)
            //    {
            //        string TEST = l[0];
            //    }

            //}


            ////</div>
            //if (Lead.Equals(""))
            //{
            //    List<string> strResult = LeadFilter.ProcessLead(Content);
            //    Lead = StripHTML(strResult[0]);
            //    Content = HttpUtility.HtmlEncode(strResult[1]);
            //}
            //else
            //{
            //    Content = HttpUtility.HtmlEncode(Content.Replace(Lead, ""));
            //}
            //replace tag 
            //example : <div> replace by none character
           
            //if (tf.Length == 0)
            //    tf = cf.Substring(0, Math.Min(80, cf.Length));
            #endregion
                    
          
            #region Create FTS
            var b = new StringBuilder();
            b.AppendLine(tf).Append(lf).AppendLine(cf);
            var FTS = b.ToString();
            #endregion
        
            return true;
        }
                       
        #endregion

        #region helpers

        public string CleanHtml(string s, string tag, string att)
        {
            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            doc.LoadHtml(s);
            //remove tag introdue trong cafef
            RemoveTagIntrodue(doc, tag, "class", "intro");
            //remove other tag
            RemoveTag(doc, tag);
           
            //test
            StringWriter writerTag = new StringWriter();
            doc.Save(writerTag);

            string resultTag = writerTag.ToString();



            RemoveAttribute(doc, att);
            StringWriter writer = new StringWriter();
            doc.Save(writer);

            string result = writer.ToString();
            if (string.IsNullOrEmpty(result)) result = s;

            writer.Close();

            return result;
        }

        public string CleanHtmlContentCafefLand(string s, string tag, string att)
        {
            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            doc.LoadHtml(s);
            //remove tag introdue trong cafef
          //  RemoveTagIntrodue(doc, "div", "class", "intro");
            //remove other tag
            RemoveTag(doc, tag);

            //test
            StringWriter writerTag = new StringWriter();
            doc.Save(writerTag);

            string resultTag = writerTag.ToString();



            RemoveAttribute(doc, att);
            StringWriter writer = new StringWriter();
            doc.Save(writer);

            string result = writer.ToString();
            if (string.IsNullOrEmpty(result)) result = s;

            writer.Close();

            return result;
        }

        /// <summary>
        /// Author: TungNX
        /// Created Date:
        /// Remove tag introdue
        /// </summary>
        /// <param name="doc"></param>
        /// <param name="tags"></param>
        private void RemoveTagIntrodue(HtmlAgilityPack.HtmlDocument doc, string tags, string att, string attvalue)
        {
            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//" + tags);
            if (nodes == null)
                return;
            foreach (HtmlNode node in nodes)
            {
                if (node.ParentNode != null && node.Attributes[att].Value.ToString().Contains(attvalue))
                {
                    nodes.Remove(node);
                    break;
                }    // Modifile(Add "return;")  VietLV (Date 01/06/2011)
                 //   node.ParentNode.RemoveChild(node);
            }
        }
        /// <summary>
        /// Author: TungNX
        /// Created Date : 
        /// Remove thẻ introdue trong cafef
        /// </summary>
        /// <param name="s"></param>
        /// <param name="tag"></param>
        /// <param name="att"></param>
        /// <returns></returns>
        public string RemoveTagIntrodue(string s, string tag, string att, string attvalue)
        {
            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            doc.LoadHtml(s);
            RemoveTagIntrodue(doc, tag, att, attvalue);
                             
            StringWriter writer = new StringWriter();
            doc.Save(writer);

            string result = writer.ToString();
            if (string.IsNullOrEmpty(result)) result = s;

            writer.Close();

            return result;
        }

        private void RemoveAttribute(HtmlAgilityPack.HtmlDocument doc, string attributes)
        {
            StringBuilder sb = new StringBuilder("//*[");
            char[] splitter = { '|' };
            string[] atts = attributes.Split(splitter);
            for (int i = 0; i < atts.Length; i++)
            {
                if (i > 0)
                    sb.Append("|");
                sb.Append("@");
                sb.Append(atts[i]);
            }
            sb.Append("]");
            string XPath = sb.ToString();
            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(XPath);
            if (nodes == null)
                return;
            foreach (HtmlNode node in nodes)
            {
                foreach (string attName in atts)
                {
                    HtmlAttribute att = node.Attributes[attName];
                    if (att != null)
                        node.Attributes.Remove(att);
                }
            }
        }   

        private void RemoveTag(HtmlAgilityPack.HtmlDocument doc, string tags)
        {
            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//" + tags);
            if (nodes == null)
                return;
            foreach (HtmlNode node in nodes)
            {
                if (node.ParentNode != null)
                    node.ParentNode.RemoveChild(node);
            }
        }
        public string StripTagsAndAttributes(string Input, string AllowedTags)
        {
            /* Remove all unwanted tags first */
            string Output = StripTags(Input, AllowedTags);

            /* Lambda functions */
            MatchEvaluator HrefMatch = m => m.Groups[1].Value + "href..;,;.." + m.Groups[2].Value;
            MatchEvaluator ClassMatch = m => m.Groups[1].Value + "class..;,;.." + m.Groups[2].Value;
            MatchEvaluator UnsafeMatch = m => m.Groups[1].Value + m.Groups[4].Value;

            /* Allow the "href" attribute */
            Output = new Regex("(<a.*)href=(.*>)").Replace(Output, HrefMatch);

            /* Allow the "class" attribute */
            Output = new Regex("(<a.*)class=(.*>)").Replace(Output, ClassMatch);

            /* Remove unsafe attributes in any of the remaining tags */
            //Output = new Regex(@"(<.*) .*=(\'|\""|\w)[\w|.|(|)]*(\'|\""|\w)(.*>)").Replace(Output, UnsafeMatch);

            /* Return the allowed tags to their proper form */
            Output = ReplaceAll(Output, "..;,;..", "=");

            return Output;
        }
        public string StripTags(string Input, string allowtag)
        {
            char[] splitter = { '|' };
            string[] AllowedTags = allowtag.Split(splitter);

            Regex StripHTMLExp = new Regex(@"(<\/?[^>]+>)");
            string Output = Input;

            foreach (Match Tag in StripHTMLExp.Matches(Input))
            {
                string HTMLTag = Tag.Value.ToLower();
                bool IsAllowed = false;

                foreach (string AllowedTag in AllowedTags)
                {
                    int offset = -1;

                    // Determine if it is an allowed tag
                    // "<tag>" , "<tag " and "</tag"
                    if (offset != 0) offset = HTMLTag.IndexOf('<' + AllowedTag + '>');
                    if (offset != 0) offset = HTMLTag.IndexOf('<' + AllowedTag + ' ');
                    if (offset != 0) offset = HTMLTag.IndexOf("</" + AllowedTag);

                    // If it matched any of the above the tag is allowed
                    if (offset == 0)
                    {
                        IsAllowed = true;
                        break;
                    }
                }
                // Remove tags that are not allowed
                if (!IsAllowed) Output = ReplaceFirst(Output, Tag.Value, "");

            }

            //remove unwanted

            return Output;
        }
        private string ReplaceFirst(string haystack, string needle, string replacement)
        {
            int pos = haystack.IndexOf(needle);
            if (pos < 0) return haystack;
            return haystack.Substring(0, pos) + replacement + haystack.Substring(pos + needle.Length);
        }
        private string ReplaceAll(string haystack, string needle, string replacement)
        {
            int pos;
            // Avoid a possible infinite loop
            if (needle == replacement) return haystack;
            while ((pos = haystack.IndexOf(needle)) > 0)
                haystack = haystack.Substring(0, pos) + replacement + haystack.Substring(pos + needle.Length);
            return haystack;
        }
        private string RemoveHTML(string htmlString)
        {
            string regularExpresion = "<[^<>]*>";
            Regex regex = new Regex(regularExpresion, RegexOptions.IgnoreCase | RegexOptions.Multiline);
            return regex.Replace(htmlString, "").Trim();
        }
        private string RemoveBetween(string s, string begin, string end)
        {

            Regex regex = new Regex(string.Format("\\{0}.*?\\{1}", begin, end));
            return regex.Replace(s, string.Empty);
        }
        public static string[] GetStringInBetween(string strBegin, string strEnd, string strSource, bool includeBegin, bool includeEnd)
        {

            string[] result = { "", "" };

            int iIndexOfBegin = strSource.IndexOf(strBegin);

            if (iIndexOfBegin != -1)
            {

                // include the Begin string if desired

                if (includeBegin)

                    iIndexOfBegin -= strBegin.Length;

                strSource = strSource.Substring(iIndexOfBegin

                    + strBegin.Length);

                int iEnd = strSource.IndexOf(strEnd);

                if (iEnd != -1)
                {

                    // include the End string if desired

                    if (includeEnd)

                        iEnd += strEnd.Length;

                    result[0] = strSource.Substring(0, iEnd);

                    // advance beyond this segment

                    if (iEnd + strEnd.Length < strSource.Length)

                        result[1] = strSource.Substring(iEnd

                            + strEnd.Length);

                }

            }

            else

                // stay where we are

                result[1] = strSource;

            return result;

        }
        public static string StripHTML(string htmlString)
        {
            string pattern = @"<(.|\n)*?>";
            return Regex.Replace(htmlString, pattern, string.Empty);

        }
        #endregion

    }
}